# encoding: utf-8
"""
@author: youfeng
@file: qingdao_new_register.py
@time: 2018/9/1 上午9:07
"""
import copy
import sys

import click
from pyquery import PyQuery

sys.path.append("..")
sys.path.append("../..")
sys.path.append("../../..")
from proxy.proxy_type import ProxyType

from common.date_util import DateUtil
from ext.task_base import TaskBase
from logger import AppLogger
from mq.mq_factory import MQFactory

logger = AppLogger('qingdao_new_register.log').get_logger()


class QingDaoNewRegister(TaskBase):
    __START_URL = "http://credit.qingdao.gov.cn/credit.permissionList.dhtml?id=&keyword=&page={}&node="

    # 新gsxt的线索队列
    __GSXT_DATA = {
        "province": u"山东",
        "company": None,
        "source_url": u"credit.qingdao.gov.cn",
        "register_code": None,
    }

    def __init__(self, before_date, log):
        super(QingDaoNewRegister, self).__init__(log)
        self._reset_beanstalk_handler(MQFactory.get_gsxt_clue_beanstalk_handler(log))
        self.__before_date = DateUtil.get_before_day(before_date)
        self.log.info("获得 {} 之后的数据...".format(self.__before_date))
        self.__set_headers()

    def __set_headers(self):
        headers = {
            "Host": "credit.qingdao.gov.cn",
            "Connection": "keep-alive",
            "Cache-Control": "max-age=0",
            "Upgrade-Insecure-Requests": "1",
            "DNT": "1",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,ja;q=0.6",
        }
        self._proxy_requests.set_header(headers)

    def __get_total_page(self, jq):
        try:
            page_num = jq.find(".pagination").find("A").eq(-2).text()
            return int(page_num)
        except Exception as e:
            self.log.error("获取页码数异常: ")
            self.log.exception(e)
        return -1

    def __get_page_data(self, url):
        resp = self._proxy_requests.get(url, proxy_type=ProxyType.KUNPENG_DYNAMIC)
        if resp is None:
            return -1

        jq = PyQuery(resp.text, parser="html")
        tr_list = jq.find(".table.table-hover.table-condensed").find("tr")
        if tr_list is None:
            return -1

        # 遍历所有的tr
        for tr_item in tr_list.items():
            td_list = tr_item.find("td")
            if not isinstance(td_list, PyQuery) or len(td_list) < 4:
                continue

            text = td_list.eq(1).find("a").attr("title")
            if not isinstance(text, basestring):
                continue
            company = text.strip()
            if len(company) <= 4:
                continue

            text = td_list.eq(3).text()
            if isinstance(text, basestring):
                date = text.strip()
                if date < self.__before_date:
                    return -2
            else:
                date = None

            gsxt_data = copy.deepcopy(self.__GSXT_DATA)
            gsxt_data["company"] = company
            self.push_gsxt_clue_data(gsxt_data)

            if date is not None:
                self.log.info("当前采集到新注册企业: date = {} company = {}".format(date, company))
            else:
                self.log.info("当前采集到新注册企业: company = {}".format(company))
        return self.__get_total_page(jq)

    def start(self, *args, **kwargs):
        total_page = self.__get_page_data(self.__START_URL.format(1))
        if total_page <= 0:
            self.log.warn("获取页码数信息异常，不进行遍历: total_page = {}".format(total_page))
            return

        for page in xrange(2, total_page + 1):
            try:
                self.log.info("当前采集页面: page = {}".format(page))
                result = self.__get_page_data(self.__START_URL.format(page))
                if result == -2:
                    self.log.info("采到时间截止，停止采集..")
                    break
                self.log.info("当前页面采集完成: page = {}".format(page))
            except Exception as e:
                self.log.error("当前页面采集失败: page = {}".format(page))
                self.log.exception(e)

        self.log.info("成功退出采集程序...")


@click.command()
@click.option('--before_date',
              default=7,
              type=int,
              help='采集截止时间')
def main(before_date):
    try:
        QingDaoNewRegister(before_date, logger)()
    except Exception as e:
        logger.error("采集异常退出: ")
        logger.exception(e)


if __name__ == '__main__':
    main()
